{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JNSdeDvgpdng"
      },
      "source": [
        "# Notebook Setup\n",
        "<a target=\"_blank\" href=\"https://colab.research.google.com/github/PacktPublishing/Generative-AI-Integration-Patterns-1E/blob/main/Chapter05/Batch_metadata_extraction.ipynb\">\n",
        "  <img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/>\n",
        "</a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nZzaJzUUoRbW"
      },
      "outputs": [],
      "source": [
        "#Install dependencies\n",
        "\n",
        "!pip install --upgrade google-cloud-aiplatform"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "euBE2B3DpHSQ"
      },
      "outputs": [],
      "source": [
        "#Authenticate\n",
        "!gcloud auth application-default login"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IHRIcyzsoT6J"
      },
      "outputs": [],
      "source": [
        "#General Imports\n",
        "\n",
        "import base64\n",
        "import json\n",
        "\n",
        "#GCP Imports\n",
        "import vertexai\n",
        "from vertexai.generative_models import GenerativeModel, Part, FinishReason\n",
        "import vertexai.preview.generative_models as generative_models\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "qgNuihpJqgkJ"
      },
      "outputs": [],
      "source": [
        "PROJECT = \"YOUR-GCP-PROJECT\"#@param {type:\"string\"}\n",
        "LOCATION = \"us-central1\"#@param {type:\"string\"}\n",
        "MODEL = \"gemini-1.5-pro-001\"#@param {type:\"string\"}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "91JnL54jpkkE"
      },
      "source": [
        "# Function Definitions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ioi8N6nNp4xE"
      },
      "outputs": [],
      "source": [
        "generation_config = {\n",
        "    \"max_output_tokens\": 8192,\n",
        "    \"temperature\": 0,\n",
        "    \"top_p\": 0.95,\n",
        "}\n",
        "\n",
        "safety_settings = {\n",
        "    generative_models.HarmCategory.HARM_CATEGORY_HATE_SPEECH: generative_models.HarmBlockThreshold.BLOCK_ONLY_HIGH,\n",
        "    generative_models.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: generative_models.HarmBlockThreshold.BLOCK_ONLY_HIGH,\n",
        "    generative_models.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: generative_models.HarmBlockThreshold.BLOCK_ONLY_HIGH,\n",
        "    generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT: generative_models.HarmBlockThreshold.BLOCK_ONLY_HIGH,\n",
        "}\n",
        "\n",
        "def generate(prompt_template_beggining,document,prompt_template_end):\n",
        "  vertexai.init(project=PROJECT, location=LOCATION)\n",
        "  model = GenerativeModel(MODEL)\n",
        "  responses = model.generate_content(\n",
        "      [prompt_template_beggining,document,prompt_template_end],\n",
        "      generation_config=generation_config,\n",
        "      safety_settings=safety_settings,\n",
        "      stream=False,\n",
        "  )\n",
        "  return(responses)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B7rAfCnbs22H"
      },
      "source": [
        "# Entry Point"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OeMoRv8itoCJ"
      },
      "outputs": [],
      "source": [
        "#In this section we will define the documents we are going to ingest.\n",
        "#For this example we are going to download a file directly.\n",
        "#In the case of a Cloud Function, you can leverage the object that triggered the function\n",
        "\n",
        "!wget https://s1.q4cdn.com/806093406/files/doc_downloads/2021/08/Nike10k2021.pdf -O document.pdf\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pg1e3tGBwgJ1"
      },
      "outputs": [],
      "source": [
        "#Load pdf file\n",
        "\n",
        "def get_file(file_path):\n",
        "  with open(file_path, \"rb\") as file:\n",
        "    file_data = file.read() #Read the binary data of the file\n",
        "  pdf_document = Part.from_data(mime_type=\"application/pdf\", data=file_data)\n",
        "  return pdf_document\n",
        "\n",
        "pdf_data = get_file(\"document.pdf\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ihAHrCBas5eW"
      },
      "source": [
        "# Prompt Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YkAR047_r6jL"
      },
      "outputs": [],
      "source": [
        "#In this section we define the prompt, notice that the document will be included as part of the format\n",
        "prompt_template_beggining = \"\"\"\n",
        "You are an expert business analyst specialized in 10k documents.\n",
        "\n",
        "Your task is to extract information from 10K documents, to achieve this task, follow these steps:\n",
        "\n",
        "1. Carefully analyze the document provided as context.\n",
        "2. Use the template as a reference to understand which data points to extract.\n",
        "3. Never make up information, if you don't remember, go back to read the document. If the data is not available, add \"Not available\" as the value.\n",
        "4. Return a well formed JSON document following the template using the data from the document.\n",
        "5. Always think step by step.\n",
        "<template>\n",
        "{\n",
        "    \"10k_template\": {\n",
        "        \"part_I\": {\n",
        "            \"item_1_business\": {\n",
        "                \"description\": \"Company’s main products and services, subsidiaries, and markets\",\n",
        "                \"recent_events\": null,\n",
        "                \"competition\": null,\n",
        "                \"regulations\": null,\n",
        "                \"labor_issues\": null,\n",
        "                \"operating_costs\": null,\n",
        "                \"seasonal_factors\": null,\n",
        "                \"summary\": null\n",
        "            },\n",
        "            \"item_1a_risk_factors\": {\n",
        "                \"risk_factors\": [],\n",
        "                \"summary\": null\n",
        "            },\n",
        "            \"item_1b_unresolved_staff_comments\": {\n",
        "                \"comments\": [],\n",
        "                \"summary\": null\n",
        "            },\n",
        "            \"item_2_properties\": {\n",
        "                \"properties\": [],\n",
        "                \"summary\": null\n",
        "            },\n",
        "            \"item_3_legal_proceedings\": {\n",
        "                \"proceedings\": [],\n",
        "                \"summary\": null\n",
        "            },\n",
        "            \"item_4\": {\n",
        "                \"reserved\": true\n",
        "            }\n",
        "        },\n",
        "        \"part_II\": {\n",
        "            \"item_5_market_for_registrants_common_equity\": {\n",
        "                \"market_information\": null,\n",
        "                \"number_of_holders\": null,\n",
        "                \"dividends\": null,\n",
        "                \"stock_repurchases\": null,\n",
        "                \"summary\": null\n",
        "            },\n",
        "            \"item_6_selected_financial_data\": {\n",
        "                \"financial_data\": {},\n",
        "                \"summary\": null\n",
        "            },\n",
        "            \"item_7_managements_discussion_and_analysis\": {\n",
        "                \"operations_and_financial_results\": null,\n",
        "                \"liquidity_and_capital_resources\": null,\n",
        "                \"trends_and_uncertainties\": null,\n",
        "                \"critical_accounting_judgments\": null,\n",
        "                \"off_balance_sheet_arrangements\": null,\n",
        "                \"contractual_obligations\": null,\n",
        "                \"summary\": null\n",
        "            }\n",
        "        }\n",
        "    }\n",
        "}\n",
        "\n",
        "</template>\n",
        "<document>\n",
        "\"\"\"\n",
        "prompt_template_end=\"\"\"\n",
        "</document>\n",
        "\n",
        "Response:\n",
        "\n",
        "\"\"\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LuJusiKDs7Au"
      },
      "source": [
        "# Inference"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N_YWvX5lhHZV"
      },
      "outputs": [],
      "source": [
        "result = generate(prompt_template_beggining,pdf_data,prompt_template_end)\n",
        "print(result)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fnSrhoN1tACl"
      },
      "source": [
        "# Result Postprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ggiDKZp3292w"
      },
      "outputs": [],
      "source": [
        "try:\n",
        " #Remove backticks if present\n",
        " text = result.text\n",
        " clean_text = text.replace('```json', '', 1).replace('```', '', 1)\n",
        " data = json.loads(clean_text)\n",
        " print(data['10k_template'])\n",
        " #Ingest the data in a database, this can be a full ingestion in the case of a document DB or may require additional post processing.\n",
        "except json.JSONDecodeError:\n",
        " print(\"Error: Invalid JSON data\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9Vy0H0pptCCF"
      },
      "source": [
        "# Result Presentation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bFYYaJ_EoT9h"
      },
      "outputs": [],
      "source": [
        "#In this case we are not exposing the results as they are intended to be ingested, but this is an example on how to show the results from the JSON document\n",
        "\n",
        "def traverse_json(data, path=\"\", indent=\"\"):\n",
        " \"\"\"\n",
        " This function recursively traverses a JSON object and prints\n",
        " key-value pairs based on the provided path.\n",
        "\n",
        " Args:\n",
        "     data: The JSON object to traverse.\n",
        "     path: A string representing the current path within the JSON object (optional).\n",
        "     indent: A string representing indentation for better readability (optional).\n",
        " \"\"\"\n",
        " if isinstance(data, dict):\n",
        "   for key, value in data.items():\n",
        "     new_path = f\"{path}/{key}\" if path else key\n",
        "     print(f\"{indent}{new_path}:\")\n",
        "     traverse_json(value, new_path, indent + \"  \")\n",
        " elif isinstance(data, list):\n",
        "   for i, item in enumerate(data):\n",
        "     new_path = f\"{path}/{i}\" if path else str(i)\n",
        "     print(f\"{indent}{new_path}:\")\n",
        "     traverse_json(item, new_path, indent + \"  \")\n",
        " else:\n",
        "   # Handle other data types (strings, numbers, etc.)\n",
        "   print(f\"{indent}{path}: {data}\")\n",
        "\n",
        "\n",
        "\n",
        "# Print the entire JSON structure with indentation\n",
        "traverse_json(data)\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}